package cn.q3c.download.spider;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import cn.keyvalue.ccf.common.object.Hyperlink;
import cn.keyvalue.ccf.common.object.ListPage;
import cn.keyvalue.ccf.common.util.ObjectUtils;
import cn.q3c.download.base.BaseSpider;

/**
 * 惠惠网首页刷更新 http://www.huihui.cn/all?page=1
 * 
 * 前3页，截至到上次抓取的
 * 
 * @author wangxinning
 * 
 */
public class HuihuiIndexSpider extends BaseSpider {

	private static Logger logger = Logger.getLogger(HuihuiIndexSpider.class);
	
	/**
	 * @param args
	 * @throws IOException
	 */
	public static void main(String[] args) throws IOException {

		HuihuiIndexSpider smzdmSpider = new HuihuiIndexSpider();
		smzdmSpider.getSpiderHyperlinkList();
		

	}

	public List<ListPage> getSpiderHyperlinkList() {

		List<ListPage> retList = new ArrayList<ListPage>();

		for (int z = this.getConfigProject().getFrompageno(); z <= this.getConfigProject().getMaxpageno(); z++) {
			try {
				String url = StringUtils.replace(this.getConfigProject().getUrls(), "{pageno}",
						Integer.toString(z));
				Document doc = Jsoup.connect(url).timeout((int)DateUtils.MILLIS_PER_MINUTE).get();
				Elements elements = doc.select("h3");

				if (elements != null && !elements.isEmpty()) {
					
					ListPage listPage = new ListPage();
					listPage.setUrl(url);
					
					for (int i = 0; i < elements.size(); i++) {
						String href = elements.get(i).select("a")
								.attr("abs:href");
						
						if( StringUtils.isBlank(href) )
						{
							continue;
						}
						
						if (!StringUtils.equals(href, lastUrl)) {

							Hyperlink hyperlink = new Hyperlink();
							hyperlink.setName(elements.get(i).select("a")
									.text());
							hyperlink.setTitle(StringUtils.trimToEmpty(elements.get(i).select("a")
									.attr("title")));
							hyperlink.setUrl(href);

							//retList.add(hyperlink);
							listPage.addSubHyperlink(hyperlink);
						}
						else
						{
							if( !ObjectUtils.isEmptyList(retList) )
							{
								lastUrl = retList.get(0).getUrl();
							}
							return retList;
						}
					}
					
					if( !ObjectUtils.isEmptyList(listPage.getHyperlinkList()) )
					{
						retList.add(listPage);
					}
				}
			} catch (Exception e) {
				logger.error("生成惠惠网抓取链接时出现异常", e);
			}
		}

		if( !ObjectUtils.isEmptyList(retList) )
		{
			lastUrl = retList.get(0).getUrl();//瀑布式，第一个链接做标记
		}
		
		return retList;
	}

	public String getCharset() {
		// TODO Auto-generated method stub
		return "GBK";
	}

}
